/*
 * data_reader.hpp
 *
 *  Created on: 16.10.2012
 *      Author: work
 */

#ifndef DATA_READER_HPP_
#define DATA_READER_HPP_

#include <string>
#include <memory>
//#include <sstream>
#include <fstream>
#include <curl/curl.h>
#include <functional>
#include <algorithm>
#include <boost/algorithm/string.hpp>

#include "document.hpp"
#include "token.hpp"
#include "algorithm.hpp"

#include "json/reader.h"
#include "json/writer.h"

#include "stem/stemming.h"
#include "stem/russian_stem.h"
#include "stem/english_stem.h"

namespace mpei {

#define freq_min 2.0
#define freq_max 1000.0

static int writer(char *data, size_t size, size_t nmemb, std::string *buffer) {
	int result = 0;
	if (buffer != NULL) {
		buffer->append(data, size * nmemb);
		result = size * nmemb;
	}

	return result;
}

typedef Document<token_t> document_t;

template<typename _S = std::string>
class DataReader {

public:
	typedef std::shared_ptr<DataReader<_S> > shared_ptr;
	typedef std::vector<_S> container;
	typedef typename std::vector<_S>::iterator iterator;

	DataReader(const _S & url) :
			frequencyMin(freq_min), frequencyMax(freq_max), headers(NULL), source(
					url) {
		curl_slist_append(headers, "Accept: application/json");
		curl_slist_append(headers, "Content-Type: application/json");
		curl_slist_append(headers, "charsets: utf-8");
		curl = curl_easy_init();
	}

	virtual ~DataReader() {
	}

	void getClasses(container & _container) {

		_S url_ = "getClasses";
		if (!curl) {
			throw std::runtime_error("curl");
		}
		CURLcode res;
		_S buffer;
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_URL, (source + url_).c_str());
		curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writer);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
		res = curl_easy_perform(curl);

		if (CURLE_OK != res) {
			throw std::runtime_error("CURLE not OK");
		}
		char *ct;
		res = curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct);
		if ((CURLE_OK == res) && ct) {
			if (buffer.empty()) {
				throw std::runtime_error("empty buffer");
			}
			Json::Value root_;
			Json::Reader reader_;
			if (reader_.parse(buffer, root_)) {
				for (std::uint64_t i = 0; i < root_.size(); i++) {
//					std::cout << "JSON array values: "
//							<< root_[i]["name"].asString() << std::endl;
					_container.push_back(root_[i]["name"].asString());
				}
			}
			buffer.clear();
		}
	}

	template<typename _T>
	size_t getDocuments(const double procent, const _S & class_name,
			_T & doc_class, _T & other) {
		_S class_name_;
		size_t size_;
		char* p = curl_escape(class_name.c_str(), (int) class_name.size());
		if (!p) {
			throw std::runtime_error("unable to escape the string"); //we got an error
		} else {
			class_name_ = p;
			curl_free(p);
		};
		_S url = "getDocuments?class=" + class_name_;

//		std::cout << (source + url).c_str() << std::endl;
		if (!curl) {
			throw std::runtime_error("curl is null");
		}
		CURLcode res;
		_S buffer_docs;
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_URL, (source + url).c_str());
		curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writer);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_docs);
		res = curl_easy_perform(curl);

		if (CURLE_OK != res) {
			throw std::runtime_error("CURLE not OK");
		}
		char *ct;
		res = curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct);
		if ((CURLE_OK == res) && ct) {
			if (buffer_docs.empty()) {
				throw std::runtime_error("empty buffer");
			}
			Json::Value root_;
			Json::Reader reader_;
			if (reader_.parse(buffer_docs, root_)) {
				size_ = root_.size();
				size_t j = size_ * procent;
//				std::cout << "data size: " << size_ << " test data: " << j
//						<< std::endl;

				//test data
				for (std::uint64_t i = 0; i < j; i++) {
					auto doc_ = document_t::shared_ptr(
							new document_t(root_[i]["name"].asString()));

#ifdef DEBUG
					std::cout << root_[i]["annotation"].asString()
					<< std::endl;
#endif
					other->insert(doc_);
					doc_->setAnnotation(root_[i]["annotation"].asString());
				}

				for (auto doc_ : other->get()) {
					std::ofstream data_file_(doc_->getDocumentClass()->getClassName() + "test",
									std::ios::app);
					if (!getTokens(doc_->getAnnotation(), doc_,data_file_)) {
						std::cout << "getTokens error" << doc_->getAnnotation()
								<< std::endl;
						continue;
					}
				}

				//training data
				for (std::uint64_t i = j; i < root_.size(); i++) {
					auto doc_ = document_t::shared_ptr(
							new document_t(root_[i]["name"].asString()));

#ifdef DEBUG
					std::cout << root_[i]["annotation"].asString()
					<< std::endl;
#endif
					doc_class->insert(doc_);
					doc_->setAnnotation(root_[i]["annotation"].asString());
//					std::cout << root_[i]["name"].asString() << std::endl;
//					std::cout << root_[i]["annotation"].asString() << std::endl;
				}
				for (auto doc_ : doc_class->get()) {
					std::ofstream data_file_(doc_->getDocumentClass()->getClassName(),
														std::ios::app);
					if (!getTokens(doc_->getAnnotation(), doc_,data_file_)) {
						std::cout << "getTokens error" << doc_->getAnnotation()
								<< std::endl;
						continue;
					}
				}
				//Write json files
//				Json::StyledStreamWriter writer;
//				std::ofstream data_file_(class_name);
////				std::cout << class_data << std::endl;
//				writer.write(data_file_, class_data);
			}
		}
		return size_;
	}

	template<typename _T>
	bool getTokens(const _S & text, _T & doc,std::ofstream & data_file_ ) {
		_S tmp_word_;
		char* p = curl_escape(text.c_str(), (int) text.size());
		if (!p) {
			throw std::runtime_error("unable to escape the string"); //we got an error
		} else {
			tmp_word_ = p;
			curl_free(p);
		};

		_S site_ = "http://translate.yandex.net/api/v1/tr.json/detect?";
		site_ += "text=" + tmp_word_;

		CURLcode res;
		_S buffer_tokens;
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
#ifdef DEBUG
		std::cout << site_ << std::endl;
#endif

		curl_easy_setopt(curl, CURLOPT_URL, site_.c_str());
		curl_easy_setopt(curl, CURLOPT_HTTPGET, 1);
		curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writer);
		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer_tokens);
		res = curl_easy_perform(curl);

		if (CURLE_OK != res) {
			throw std::runtime_error("CURLE not OK");
		}
		char *ct;
		res = curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &ct);
		std::ifstream stop_file_;
		if ((CURLE_OK != res) && !ct) {
			std::cout << "not CURLINFO_CONTENT_TYPE" << std::endl;
		}
		if (buffer_tokens.empty()) {
//				throw std::runtime_error("empty buffer");
			std::cout << "buffer_tokens.empty()" << std::endl;
			return false;
		}
		Json::Value root_;
		Json::Reader reader_;
//			stem = std::shared_ptr<stemming::stem<>>(new stemming::stem<>());
		if (!reader_.parse(buffer_tokens, root_)) {
			std::cout << "!reader_.parse" << std::endl;
			return false;
		}
//				std::cout << buffer_tokens << std::endl;
		if (root_["code"].asUInt() != 200) {
			std::cout << "root_[code]" << std::endl;
			return false;
		}
		_S lang_ = root_["lang"].asString();
		if (!lang_.compare("ru")) {
			stop_file_.open("src/stop_words/RussianStopWords.txt");
			if (!stop_file_.is_open()) {
//							throw std::runtime_error("no russian stop_words");
				std::cout << "!stop_file_.is_open()" << std::endl;
				return false;
			}
#ifdef DEBUG
			std::cout << "russian" << std::endl;
#endif
			stem = std::shared_ptr<stemming::stem<std::wstring>>(
					new stemming::russian_stem<std::wstring>());

		} else if (!lang_.compare("en")) {
			stop_file_.open("src/stop_words/EnglishStopWords.txt");
			if (!stop_file_.is_open()) {
//							throw std::runtime_error("no russian stop_words");
				std::cout << "!stop_file_.is_open()" << std::endl;
				return false;
			}
#ifdef DEBUG
			std::cout << "english" << std::endl;
#endif
			stem = std::shared_ptr<stemming::stem<std::wstring>>(
					new stemming::english_stem<std::wstring>());

		} else {
//						throw std::runtime_error("undef language");
			std::cout << "no lang" << std::endl;
			return false;
		}
		_S stop_word_;
		container stop_words_;
		while (stop_file_ >> stop_word_) {
//			std::cout << stop_word_ << std::endl;
			stop_words_.push_back(stop_word_);
		}
		container strs;
		iterator it1, it2;
		boost::split(strs, text,
				boost::is_any_of("[]()\?/\"\' ;:-,.\n\r\t123456789_&%#@~*"));
//		for( auto word : strs) {
//			boost::algorithm::to_lower(word);
//		}

		strs.erase(
				std::remove_if(strs.begin(), strs.end(), [&](_S token)->bool {
					it2 = std::find_if(stop_words_.begin(), stop_words_.end(),
							[&](_S word)->bool {
								if(!token.compare(word) || token.empty()) {
									return true;
								}
								return false;
							});
					if (it2 != stop_words_.end()) {
						return true;
					}
					return false;
				}), strs.end());
//		for (auto token : strs) {
//			std::cout << token << std::endl;
//		}

		auto converter = ConverterUtf8();
		Json::Value document_node;
		for (auto token : strs) {
			std::wstring wstr1 = converter(token);
			stem.operator *()(wstr1);
			_S str2 = converter(wstr1);
#ifdef DEBUG
			std::cout << token << std::endl;
			std::cout << str2 << std::endl;
#endif
			auto token_ = token_t::shared_ptr(new token_t(str2));
			doc->insert(token_);
			//TODO add Token_t
		}
		doc->get().erase(
				std::remove_if(doc->get().begin(), doc->get().end(),
						[&](token_t::shared_ptr token) {
							if(token->getFrequency() < frequencyMin ||
									token->getFrequency() > frequencyMax) {
								return true;
							}
							return false;
						}), doc->get().end());

		for (auto token_ : doc->get()) {
			doc->calculate_tf(token_);
			Json::Value js_token_;
			js_token_["token"] = token_->data();
//			js_token_["freq"] = token_->getFrequency();
			js_token_["freq"] = token_->getTf();
			document_node.append(js_token_);
		}
		if (document_node.empty() || document_node.isNull()) {
			return false;
		}
		Json::Value w_root;
		w_root["document"] = doc->getDocumentName();
		w_root["class"] = doc->getDocumentClass()->getClassName();
		w_root["tokens"] = document_node;

//		w_root[doc->getDocumentName()] = document_node;
		class_data.append(w_root);
		std::cout << w_root << std::endl;
		//open with add lines

		Json::FastWriter writer;
		std::cout << writer.write(w_root) << std::endl;
		data_file_ << writer.write(w_root) << std::endl;
		return true;
	}

	bool getClass(const _S & str) {
		Json::Reader reader_;
		Json::Value root_;
		std::ifstream file_name_(str);
		if (!reader_.parse(file_name_, root_)) {
			return false;
		}
		for (std::uint64_t i = 0; i < root_.size(); i++) {
//			std::cout << root_[i].toStyledString() << std::endl;
		}
		return true;
	}

	void setFrequencyInterval(token_t::value_type min,
			token_t::value_type max) {
		frequencyMin = min;
		frequencyMax = max;
	}
private:
	token_t::value_type frequencyMin;
	token_t::value_type frequencyMax;
	struct curl_slist *headers;
	std::shared_ptr<stemming::stem<std::wstring>> stem;
	CURL *curl;
	Json::Value class_data;
	_S source;

}
;

typedef DataReader<> data_reader_t;

}  // namespace mpei

#endif /* DATA_READER_HPP_ */
